/*
    SUSAN® - Sole of Unix Save ANything

   
      Copyright (C) 2011-2016 Skyatlas Co. LTD

   

   
*/
/*
 * Kern Sibbald, November MM
 */
/**
 * @file
 * Susan File Daemon restore.c Restorefiles.
 */

#include "include/susan.h"
#include "filed/filed.h"
#include "filed/filed_globals.h"
#include "filed/jcr_private.h"
#include "filed/compression.h"
#include "filed/crypto.h"
#include "filed/restore.h"
#include "filed/verify.h"
#include "include/ch.h"
#include "findlib/create_file.h"
#include "findlib/attribs.h"
#include "findlib/find.h"
#include "lib/berrno.h"
#include "lib/bget_msg.h"
#include "lib/bnet.h"
#include "lib/bsock.h"
#include "lib/edit.h"
#include "lib/parse_conf.h"
#include "include/make_unique.h"

#ifdef HAVE_WIN32
#include "win32/findlib/win32.h"
#endif

#if defined(HAVE_DARWIN_OS)
#include <sys/attr.h>
#endif

namespace filedaemon {

#if defined(HAVE_DARWIN_OS)
const bool have_darwin_os = true;
#else
const bool have_darwin_os = false;
#endif

#if defined(HAVE_CRYPTO)
const bool have_crypto = true;
#else
const bool have_crypto = false;
#endif

#if defined(HAVE_ACL)
const bool have_acl = true;
#else
const bool have_acl = false;
#endif

#if defined(HAVE_SHA2)
const bool have_sha2 = true;
#else
const bool have_sha2 = false;
#endif

#if defined(HAVE_XATTR)
const bool have_xattr = true;
#else
const bool have_xattr = false;
#endif

/**
 * Data received from Storage Daemon
 */
static char rec_header[] = "rechdr %ld %ld %ld %ld %ld";

/**
 * Forward referenced functions
 */
#if defined(HAVE_LIBZ)
const bool have_libz = true;
#else
const bool have_libz = false;
#endif
#if defined(HAVE_LZO)
const bool have_lzo = true;
#else
const bool have_lzo = false;
#endif
const bool have_fastlz = true;

static void FreeSignature(r_ctx& rctx);
static bool ClosePreviousStream(JobControlRecord* jcr, r_ctx& rctx);

int32_t ExtractData(JobControlRecord* jcr,
                    SusanWinFilePacket* bfd,
                    POOLMEM* buf,
                    int32_t buflen,
                    uint64_t* addr,
                    char* flags,
                    int32_t stream,
                    RestoreCipherContext* cipher_ctx);

/**
 * Close a bfd check that we are at the expected file offset.
 * Makes use of some code from SetAttributes().
 */
static int BcloseChksize(JobControlRecord* jcr,
                         SusanWinFilePacket* bfd,
                         boffset_t osize)
{
  char ec1[50], ec2[50];
  boffset_t fsize;

  fsize = blseek(bfd, 0, SEEK_CUR);
  bclose(bfd);
  if (fsize > 0 && fsize != osize) {
    Qmsg3(jcr, M_WARNING, 0,
          _("Size of data or stream of %s not correct. Original %s, restored "
            "%s.\n"),
          jcr->impl->last_fname, edit_uint64(osize, ec1),
          edit_uint64(fsize, ec2));
    return -1;
  }
  return 0;
}

static inline bool RestoreFinderinfo(JobControlRecord* jcr,
                                     POOLMEM* buf,
                                     int32_t buflen)
{
#ifdef HAVE_DARWIN_OS
  struct attrlist attrList;

  memset(&attrList, 0, sizeof(attrList));
  attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
  attrList.commonattr = ATTR_CMN_FNDRINFO;

  Dmsg0(130, "Restoring Finder Info\n");
  SetBit(FO_HFSPLUS, jcr->impl->ff->flags);
  if (buflen != 32) {
    Jmsg(jcr, M_WARNING, 0,
         _("Invalid length of Finder Info (got %d, not 32)\n"), buflen);
    return false;
  }

  if (setattrlist(jcr->impl->last_fname, &attrList, buf, buflen, 0) != 0) {
    Jmsg(jcr, M_WARNING, 0, _("Could not set Finder Info on %s\n"),
         jcr->impl->last_fname);
    return false;
  }

  return true;
#else
  return true;
#endif
}

/**
 * Cleanup of delayed restore stack with streams for later processing.
 */
static inline void DropDelayedDataStreams(r_ctx& rctx, bool reuse)
{
  DelayedDataStream* dds = nullptr;

  if (!rctx.delayed_streams || rctx.delayed_streams->empty()) { return; }

  foreach_alist (dds, rctx.delayed_streams) {
    free(dds->content);
  }

  rctx.delayed_streams->destroy();
  if (reuse) { rctx.delayed_streams->init(10, owned_by_alist); }
}

/**
 * Push a data stream onto the delayed restore stack for later processing.
 */
static inline void PushDelayedDataStream(r_ctx& rctx, SusanSocket* sd)
{
  DelayedDataStream* dds;

  if (!rctx.delayed_streams) {
    rctx.delayed_streams = new alist(10, owned_by_alist);
  }

  dds = (DelayedDataStream*)malloc(sizeof(DelayedDataStream));
  dds->stream = rctx.stream;
  dds->content = (char*)malloc(sd->message_length);
  memcpy(dds->content, sd->msg, sd->message_length);
  dds->content_length = sd->message_length;

  rctx.delayed_streams->append(dds);
}

/**
 * Perform a restore of an ACL using the stream received.
 * This can either be a delayed restore or direct restore.
 */
static inline bool do_reStoreAcl(JobControlRecord* jcr,
                                 int stream,
                                 char* content,
                                 uint32_t content_length)

{
  bacl_exit_code retval;

  jcr->impl->acl_data->last_fname = jcr->impl->last_fname;
  switch (stream) {
    case STREAM_ACL_PLUGIN:
      retval = plugin_parse_acl_streams(jcr, jcr->impl->acl_data.get(), stream,
                                        content, content_length);
      break;
    default:
      retval = parse_acl_streams(jcr, jcr->impl->acl_data.get(), stream,
                                 content, content_length);
      break;
  }

  switch (retval) {
    case bacl_exit_fatal:
      return false;
    case bacl_exit_error:
      /*
       * Non-fatal errors, count them and when the number is under
       * ACL_REPORT_ERR_MAX_PER_JOB print the error message set by the lower
       * level routine in jcr->errmsg.
       */
      if (jcr->impl->acl_data->u.parse->nr_errors <
          ACL_REPORT_ERR_MAX_PER_JOB) {
        Jmsg(jcr, M_WARNING, 0, "%s", jcr->errmsg);
      }
      jcr->impl->acl_data->u.parse->nr_errors++;
      break;
    case bacl_exit_ok:
      break;
  }

  return true;
}

/**
 * Perform a restore of an XATTR using the stream received.
 * This can either be a delayed restore or direct restore.
 */
static inline bool do_restore_xattr(JobControlRecord* jcr,
                                    int stream,
                                    char* content,
                                    uint32_t content_length)
{
  BxattrExitCode retval;

  jcr->impl->xattr_data->last_fname = jcr->impl->last_fname;
  switch (stream) {
    case STREAM_XATTR_PLUGIN:
      retval = PluginParseXattrStreams(jcr, jcr->impl->xattr_data.get(), stream,
                                       content, content_length);
      break;
    default:
      retval = ParseXattrStreams(jcr, jcr->impl->xattr_data.get(), stream,
                                 content, content_length);
      break;
  }

  switch (retval) {
    case BxattrExitCode::kErrorFatal:
      return false;
    case BxattrExitCode::kWarning:
      Jmsg(jcr, M_WARNING, 0, "%s", jcr->errmsg);
      break;
    case BxattrExitCode::kError:
      Jmsg(jcr, M_ERROR, 0, "%s", jcr->errmsg);
      jcr->impl->xattr_data->u.parse->nr_errors++;
      break;
    case BxattrExitCode::kSuccess:
      break;
  }

  return true;
}

/**
 * Restore any data streams that are restored after the file
 * is fully restored and has its attributes restored. Things
 * like acls and xattr are restored after we set the file
 * attributes otherwise we might clear some security flags
 * by setting the attributes.
 */
static inline bool PopDelayedDataStreams(JobControlRecord* jcr, r_ctx& rctx)
{
  DelayedDataStream* dds = nullptr;

  /*
   * See if there is anything todo.
   */
  if (!rctx.delayed_streams || rctx.delayed_streams->empty()) { return true; }

  /*
   * Only process known delayed data streams here.
   * If you start using more delayed data streams
   * be sure to add them in this loop and add the
   * proper calls here.
   *
   * Currently we support delayed data stream
   * processing for the following type of streams:
   * - *_ACL_*
   * - *_XATTR_*
   */
  foreach_alist (dds, rctx.delayed_streams) {
    switch (dds->stream) {
      case STREAM_UNIX_ACCESS_ACL:
      case STREAM_UNIX_DEFAULT_ACL:
      case STREAM_ACL_AIX_TEXT:
      case STREAM_ACL_DARWIN_ACCESS_ACL:
      case STREAM_ACL_FREEBSD_DEFAULT_ACL:
      case STREAM_ACL_FREEBSD_ACCESS_ACL:
      case STREAM_ACL_HPUX_ACL_ENTRY:
      case STREAM_ACL_IRIX_DEFAULT_ACL:
      case STREAM_ACL_IRIX_ACCESS_ACL:
      case STREAM_ACL_LINUX_DEFAULT_ACL:
      case STREAM_ACL_LINUX_ACCESS_ACL:
      case STREAM_ACL_TRU64_DEFAULT_ACL:
      case STREAM_ACL_TRU64_DEFAULT_DIR_ACL:
      case STREAM_ACL_TRU64_ACCESS_ACL:
      case STREAM_ACL_SOLARIS_ACLENT:
      case STREAM_ACL_SOLARIS_ACE:
      case STREAM_ACL_AFS_TEXT:
      case STREAM_ACL_AIX_AIXC:
      case STREAM_ACL_AIX_NFS4:
      case STREAM_ACL_FREEBSD_NFS4_ACL:
      case STREAM_ACL_HURD_DEFAULT_ACL:
      case STREAM_ACL_HURD_ACCESS_ACL:
        if (!do_reStoreAcl(jcr, dds->stream, dds->content,
                           dds->content_length)) {
          goto bail_out;
        }
        free(dds->content);
        break;
      case STREAM_XATTR_PLUGIN:
      case STREAM_XATTR_HURD:
      case STREAM_XATTR_IRIX:
      case STREAM_XATTR_TRU64:
      case STREAM_XATTR_AIX:
      case STREAM_XATTR_OPENBSD:
      case STREAM_XATTR_SOLARIS_SYS:
      case STREAM_XATTR_DARWIN:
      case STREAM_XATTR_FREEBSD:
      case STREAM_XATTR_LINUX:
      case STREAM_XATTR_NETBSD:
        if (!do_restore_xattr(jcr, dds->stream, dds->content,
                              dds->content_length)) {
          goto bail_out;
        }
        free(dds->content);
        break;
      default:
        Jmsg(jcr, M_WARNING, 0,
             _("Unknown stream=%d ignored. This shouldn't happen!\n"),
             dds->stream);
        break;
    }
  }

  /*
   * We processed the stack so we can destroy it.
   */
  rctx.delayed_streams->destroy();

  /*
   * (Re)Initialize the stack for a new use.
   */
  rctx.delayed_streams->init(10, owned_by_alist);

  return true;

bail_out:

  /*
   * Destroy the content of the stack and (re)initialize it for a new use.
   */
  DropDelayedDataStreams(rctx, true);

  return false;
}

/**
 * Restore the requested files.
 */
void DoRestore(JobControlRecord* jcr)
{
  SusanSocket* sd;
  uint32_t VolSessionId, VolSessionTime;
  int32_t file_index;
  char ec1[50];      /* Buffer printing huge values */
  uint32_t buf_size; /* client buffer size */
  int status;
  int64_t rsrc_len = 0; /* Original length of resource fork */
  r_ctx rctx;
  Attributes* attr;
  /* ***FIXME*** make configurable */
  crypto_digest_t signing_algorithm =
      have_sha2 ? CRYPTO_DIGEST_SHA256 : CRYPTO_DIGEST_SHA1;
  /*
   * The following variables keep track of "known unknowns"
   */
  int non_support_data = 0;
  int non_support_attr = 0;
  int non_support_rsrc = 0;
  int non_support_finfo = 0;
  int non_support_acl = 0;
  int non_support_progname = 0;
  int non_support_crypto = 0;
  int non_support_xattr = 0;

  rctx.jcr = jcr;

  sd = jcr->store_bsock;
  jcr->setJobStatus(JS_Running);

  LockRes(my_config);
  ClientResource* client =
      (ClientResource*)my_config->GetNextRes(R_CLIENT, NULL);
  UnlockRes(my_config);
  if (client) {
    buf_size = client->max_network_buffer_size;
  } else {
    buf_size = 0; /* use default */
  }
  if (!BnetSetBufferSize(sd, buf_size, BNET_SETBUF_WRITE)) {
    jcr->setJobStatus(JS_ErrorTerminated);
    return;
  }
  jcr->buf_size = sd->message_length;

  if (have_libz || have_lzo || have_fastlz) {
    if (!AdjustDecompressionBuffers(jcr)) { goto bail_out; }
  }

  if (have_crypto) {
    rctx.cipher_ctx.buf = GetMemory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
    if (have_darwin_os) {
      rctx.fork_cipher_ctx.buf = GetMemory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
    }
  }

  /*
   * Get a record from the Storage daemon. We are guaranteed to
   *   receive records in the following order:
   *   1. Stream record header
   *   2. Stream data (one or more of the following in the order given)
   *        a. Attributes (Unix or Win32)
   *        b. Possibly stream encryption session data (e.g., symmetric session
   * key) c. File data for the file d. Alternate data stream (e.g. Resource
   * Fork) e. Finder info f. ACLs g. XATTRs h. Possibly a cryptographic
   * signature i. Possibly MD5 or SHA1 record
   *   3. Repeat step 1
   *
   * NOTE: We keep track of two susan file descriptors:
   *   1. bfd for file data.
   *      This fd is opened for non empty files when an attribute stream is
   *      encountered and closed when we find the next attribute stream.
   *   2. fork_bfd for alternate data streams
   *      This fd is opened every time we encounter a new alternate data
   *      stream for the current file. When we find any other stream, we
   *      close it again.
   *      The expected size of the stream, fork_len, should be set when
   *      opening the fd.
   *   3. Not all the stream data records are required -- e.g. if there
   *      is no fork, there is no alternate data stream, no ACL, ...
   */
  binit(&rctx.bfd);
  binit(&rctx.forkbfd);
  attr = rctx.attr = new_attr(jcr);
  if (have_acl) {
    jcr->impl->acl_data = std::make_unique<AclData>();
    jcr->impl->acl_data->u.parse =
        (acl_parse_data_t*)malloc(sizeof(acl_parse_data_t));
    memset(jcr->impl->acl_data->u.parse, 0, sizeof(acl_parse_data_t));
  }
  if (have_xattr) {
    jcr->impl->xattr_data = std::make_unique<XattrData>();
    jcr->impl->xattr_data->u.parse =
        (xattr_parse_data_t*)malloc(sizeof(xattr_parse_data_t));
    memset(jcr->impl->xattr_data->u.parse, 0, sizeof(xattr_parse_data_t));
  }

  while (BgetMsg(sd) >= 0 && !JobCanceled(jcr)) {
    /*
     * Remember previous stream type
     */
    rctx.prev_stream = rctx.stream;

    /*
     * First we expect a Stream Record Header
     */
    if (sscanf(sd->msg, rec_header, &VolSessionId, &VolSessionTime, &file_index,
               &rctx.full_stream, &rctx.size) != 5) {
      Jmsg1(jcr, M_FATAL, 0, _("Record header scan error: %s\n"), sd->msg);
      goto bail_out;
    }
    /* Strip off new stream high bits */
    rctx.stream = rctx.full_stream & STREAMMASK_TYPE;
    Dmsg5(150, "Got hdr: Files=%d FilInx=%d size=%d Stream=%d, %s.\n",
          jcr->JobFiles, file_index, rctx.size, rctx.stream,
          stream_to_ascii(rctx.stream));

    /*
     * Now we expect the Stream Data
     */
    if (BgetMsg(sd) < 0) {
      Jmsg1(jcr, M_FATAL, 0, _("Data record error. ERR=%s\n"), sd->bstrerror());
      goto bail_out;
    }
    if (rctx.size != (uint32_t)sd->message_length) {
      Jmsg2(jcr, M_FATAL, 0, _("Actual data size %d not same as header %d\n"),
            sd->message_length, rctx.size);
      Dmsg2(50, "Actual data size %d not same as header %d\n",
            sd->message_length, rctx.size);
      goto bail_out;
    }
    Dmsg3(130, "Got stream: %s len=%d extract=%d\n",
          stream_to_ascii(rctx.stream), sd->message_length, rctx.extract);

    /*
     * If we change streams, close and reset alternate data streams
     */
    if (rctx.prev_stream != rctx.stream) {
      if (IsBopen(&rctx.forkbfd)) {
        DeallocateForkCipher(rctx);
        BcloseChksize(jcr, &rctx.forkbfd, rctx.fork_size);
      }
      /*
       * Use an impossible value and set a proper one below
       */
      rctx.fork_size = -1;
      rctx.fork_addr = 0;
    }

    /*
     * File Attributes stream
     */
    switch (rctx.stream) {
      case STREAM_UNIX_ATTRIBUTES:
      case STREAM_UNIX_ATTRIBUTES_EX:
        /*
         * if any previous stream open, close it
         */
        if (!ClosePreviousStream(jcr, rctx)) { goto bail_out; }

        /*
         * TODO: manage deleted files
         */
        if (rctx.type == FT_DELETED) { /* deleted file */
          continue;
        }
        /*
         * Restore objects should be ignored here -- they are
         * returned at the beginning of the restore.
         */
        if (IS_FT_OBJECT(rctx.type)) { continue; }

        /*
         * Unpack attributes and do sanity check them
         */
        if (!UnpackAttributesRecord(jcr, rctx.stream, sd->msg,
                                    sd->message_length, attr)) {
          goto bail_out;
        }

        Dmsg3(100, "File %s\nattrib=%s\nattribsEx=%s\n", attr->fname,
              attr->attr, attr->attrEx);
        Dmsg3(100, "=== message_length=%d attrExlen=%d msg=%s\n",
              sd->message_length, strlen(attr->attrEx), sd->msg);

        attr->data_stream = DecodeStat(attr->attr, &attr->statp,
                                       sizeof(attr->statp), &attr->LinkFI);

        if (!IsRestoreStreamSupported(attr->data_stream)) {
          if (!non_support_data++) {
            Jmsg(jcr, M_WARNING, 0,
                 _("%s stream not supported on this Client.\n"),
                 stream_to_ascii(attr->data_stream));
          }
          continue;
        }

        BuildAttrOutputFnames(jcr, attr);

        /*
         * Try to actually create the file, which returns a status telling
         * us if we need to extract or not.
         */
        jcr->impl->num_files_examined++;
        rctx.extract = false;
        status = CF_CORE; /* By default, let Susan's core handle it */

        if (jcr->IsPlugin()) {
          status = PluginCreateFile(jcr, attr, &rctx.bfd, jcr->impl->replace);
        }

        if (status == CF_CORE) {
          status = CreateFile(jcr, attr, &rctx.bfd, jcr->impl->replace);
        }
        jcr->lock();
        PmStrcpy(jcr->impl->last_fname, attr->ofname);
        jcr->impl->last_type = attr->type;
        jcr->unlock();
        Dmsg2(130, "Outfile=%s CreateFile status=%d\n", attr->ofname, status);
        switch (status) {
          case CF_ERROR:
            break;
          case CF_SKIP:
            jcr->JobFiles++;
            break;
          case CF_EXTRACT:
            /*
             * File created and we expect file data
             */
            rctx.extract = true;
            /*
             * FALLTHROUGH
             */
          case CF_CREATED:
            /*
             * File created, but there is no content
             */
            rctx.fileAddr = 0;
            PrintLsOutput(jcr, attr);

            if (have_darwin_os) {
              /*
               * Only restore the resource fork for regular files
               */
              FromBase64(&rsrc_len, attr->attrEx);
              if (attr->type == FT_REG && rsrc_len > 0) { rctx.extract = true; }

              /*
               * Do not count the resource forks as regular files being
               * restored.
               */
              if (rsrc_len == 0) { jcr->JobFiles++; }
            } else {
              jcr->JobFiles++;
            }

            if (!rctx.extract) {
              /*
               * Set attributes now because file will not be extracted
               */
              if (jcr->IsPlugin()) {
                PluginSetAttributes(jcr, attr, &rctx.bfd);
              } else {
                SetAttributes(jcr, attr, &rctx.bfd);
              }
            }
            break;
        }
        break;

      /*
       * Data stream
       */
      case STREAM_ENCRYPTED_SESSION_DATA:
        if (rctx.extract) {
          crypto_error_t cryptoerr;

          /*
           * Is this an unexpected session data entry?
           */
          if (rctx.cs) {
            Jmsg0(jcr, M_ERROR, 0,
                  _("Unexpected cryptographic session data stream.\n"));
            rctx.extract = false;
            bclose(&rctx.bfd);
            continue;
          }

          /*
           * Do we have any keys at all?
           */
          if (!jcr->impl->crypto.pki_recipients) {
            Jmsg(jcr, M_ERROR, 0,
                 _("No private decryption keys have been defined to decrypt "
                   "encrypted backup data.\n"));
            rctx.extract = false;
            bclose(&rctx.bfd);
            break;
          }

          if (jcr->impl->crypto.digest) {
            CryptoDigestFree(jcr->impl->crypto.digest);
          }
          jcr->impl->crypto.digest = crypto_digest_new(jcr, signing_algorithm);
          if (!jcr->impl->crypto.digest) {
            Jmsg0(jcr, M_FATAL, 0, _("Could not create digest.\n"));
            rctx.extract = false;
            bclose(&rctx.bfd);
            break;
          }

          /*
           * Decode and save session keys.
           */
          cryptoerr = CryptoSessionDecode(
              (uint8_t*)sd->msg, (uint32_t)sd->message_length,
              jcr->impl->crypto.pki_recipients, &rctx.cs);
          switch (cryptoerr) {
            case CRYPTO_ERROR_NONE:
              /*
               * Success
               */
              break;
            case CRYPTO_ERROR_NORECIPIENT:
              Jmsg(jcr, M_ERROR, 0,
                   _("Missing private key required to decrypt encrypted backup "
                     "data.\n"));
              break;
            case CRYPTO_ERROR_DECRYPTION:
              Jmsg(jcr, M_ERROR, 0, _("Decrypt of the session key failed.\n"));
              break;
            default:
              /*
               * Shouldn't happen
               */
              Jmsg1(jcr, M_ERROR, 0,
                    _("An error occurred while decoding encrypted session data "
                      "stream: %s\n"),
                    crypto_strerror(cryptoerr));
              break;
          }

          if (cryptoerr != CRYPTO_ERROR_NONE) {
            rctx.extract = false;
            bclose(&rctx.bfd);
            continue;
          }
        }
        break;

      case STREAM_FILE_DATA:
      case STREAM_SPARSE_DATA:
      case STREAM_WIN32_DATA:
      case STREAM_GZIP_DATA:
      case STREAM_SPARSE_GZIP_DATA:
      case STREAM_WIN32_GZIP_DATA:
      case STREAM_COMPRESSED_DATA:
      case STREAM_SPARSE_COMPRESSED_DATA:
      case STREAM_WIN32_COMPRESSED_DATA:
      case STREAM_ENCRYPTED_FILE_DATA:
      case STREAM_ENCRYPTED_WIN32_DATA:
      case STREAM_ENCRYPTED_FILE_GZIP_DATA:
      case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
      case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
      case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
        if (rctx.extract) {
          bool process_data = false;

          /*
           * Force an expected, consistent stream type here
           * First see if we need to process the data and
           * set the flags.
           */
          if (rctx.prev_stream == rctx.stream) {
            process_data = true;
          } else {
            switch (rctx.prev_stream) {
              case STREAM_UNIX_ATTRIBUTES:
              case STREAM_UNIX_ATTRIBUTES_EX:
              case STREAM_ENCRYPTED_SESSION_DATA:
                process_data = true;
                break;
              default:
                break;
            }
          }

          /*
           * If process_data is set in the test above continue here with the
           * processing of the data based on the stream type available.
           */
          if (process_data) {
            ClearAllBits(FO_MAX, rctx.flags);
            switch (rctx.stream) {
              case STREAM_SPARSE_DATA:
                SetBit(FO_SPARSE, rctx.flags);
                break;
              case STREAM_SPARSE_GZIP_DATA:
              case STREAM_SPARSE_COMPRESSED_DATA:
                SetBit(FO_SPARSE, rctx.flags);
                SetBit(FO_COMPRESS, rctx.flags);
                rctx.comp_stream = rctx.stream;
                break;
              case STREAM_GZIP_DATA:
              case STREAM_COMPRESSED_DATA:
              case STREAM_WIN32_GZIP_DATA:
              case STREAM_WIN32_COMPRESSED_DATA:
                SetBit(FO_COMPRESS, rctx.flags);
                rctx.comp_stream = rctx.stream;
                break;
              case STREAM_ENCRYPTED_FILE_GZIP_DATA:
              case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
              case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
              case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
                if (!rctx.cipher_ctx.cipher) {
                  if (!SetupDecryptionContext(rctx, rctx.cipher_ctx)) {
                    rctx.extract = false;
                    bclose(&rctx.bfd);
                    continue;
                  }
                }
                SetBit(FO_COMPRESS, rctx.flags);
                SetBit(FO_ENCRYPT, rctx.flags);
                rctx.comp_stream = rctx.stream;
                break;
              case STREAM_ENCRYPTED_FILE_DATA:
              case STREAM_ENCRYPTED_WIN32_DATA:
                if (!rctx.cipher_ctx.cipher) {
                  if (!SetupDecryptionContext(rctx, rctx.cipher_ctx)) {
                    rctx.extract = false;
                    bclose(&rctx.bfd);
                    continue;
                  }
                }
                SetBit(FO_ENCRYPT, rctx.flags);
                break;
              default:
                break;
            }

            /*
             * Check for a win32 stream type on a system without the win32 API.
             * On those we decompose the BackupWrite data.
             */
            if (is_win32_stream(rctx.stream) && !have_win32_api()) {
              SetPortableBackup(&rctx.bfd);
              /*
               * "decompose" BackupWrite data
               */
              SetBit(FO_WIN32DECOMP, rctx.flags);
            }

            if (ExtractData(jcr, &rctx.bfd, sd->msg, sd->message_length,
                            &rctx.fileAddr, rctx.flags, rctx.stream,
                            &rctx.cipher_ctx) < 0) {
              rctx.extract = false;
              bclose(&rctx.bfd);
              continue;
            }
          }
        }
        break;

      /*
       * Resource fork stream - only recorded after a file to be restored
       * Silently ignore if we cannot write - we already reported that
       */
      case STREAM_ENCRYPTED_MACOS_FORK_DATA:
      case STREAM_MACOS_FORK_DATA:
        if (have_darwin_os) {
          ClearAllBits(FO_MAX, rctx.fork_flags);
          SetBit(FO_HFSPLUS, jcr->impl->ff->flags);

          if (rctx.stream == STREAM_ENCRYPTED_MACOS_FORK_DATA) {
            SetBit(FO_ENCRYPT, rctx.fork_flags);
            if (rctx.extract && !rctx.fork_cipher_ctx.cipher) {
              if (!SetupDecryptionContext(rctx, rctx.fork_cipher_ctx)) {
                rctx.extract = false;
                bclose(&rctx.bfd);
                continue;
              }
            }
          }

          if (rctx.extract) {
            if (rctx.prev_stream != rctx.stream) {
              if (BopenRsrc(&rctx.forkbfd, jcr->impl->last_fname,
                            O_WRONLY | O_TRUNC | O_BINARY, 0) < 0) {
                Jmsg(jcr, M_WARNING, 0,
                     _("Cannot open resource fork for %s.\n"),
                     jcr->impl->last_fname);
                rctx.extract = false;
                continue;
              }

              rctx.fork_size = rsrc_len;
              Dmsg0(130, "Restoring resource fork\n");
            }

            if (ExtractData(jcr, &rctx.forkbfd, sd->msg, sd->message_length,
                            &rctx.fork_addr, rctx.fork_flags, rctx.stream,
                            &rctx.fork_cipher_ctx) < 0) {
              rctx.extract = false;
              bclose(&rctx.forkbfd);
              continue;
            }
          }
        } else {
          non_support_rsrc++;
        }
        break;

      case STREAM_HFSPLUS_ATTRIBUTES:
        if (have_darwin_os) {
          if (!RestoreFinderinfo(jcr, sd->msg, sd->message_length)) {
            continue;
          }
        } else {
          non_support_finfo++;
        }
        break;

      case STREAM_UNIX_ACCESS_ACL:
      case STREAM_UNIX_DEFAULT_ACL:
      case STREAM_ACL_AIX_TEXT:
      case STREAM_ACL_DARWIN_ACCESS_ACL:
      case STREAM_ACL_FREEBSD_DEFAULT_ACL:
      case STREAM_ACL_FREEBSD_ACCESS_ACL:
      case STREAM_ACL_HPUX_ACL_ENTRY:
      case STREAM_ACL_IRIX_DEFAULT_ACL:
      case STREAM_ACL_IRIX_ACCESS_ACL:
      case STREAM_ACL_LINUX_DEFAULT_ACL:
      case STREAM_ACL_LINUX_ACCESS_ACL:
      case STREAM_ACL_TRU64_DEFAULT_ACL:
      case STREAM_ACL_TRU64_DEFAULT_DIR_ACL:
      case STREAM_ACL_TRU64_ACCESS_ACL:
      case STREAM_ACL_SOLARIS_ACLENT:
      case STREAM_ACL_SOLARIS_ACE:
      case STREAM_ACL_AFS_TEXT:
      case STREAM_ACL_AIX_AIXC:
      case STREAM_ACL_AIX_NFS4:
      case STREAM_ACL_FREEBSD_NFS4_ACL:
      case STREAM_ACL_HURD_DEFAULT_ACL:
      case STREAM_ACL_HURD_ACCESS_ACL:
        /*
         * Do not restore ACLs when
         * a) The current file is not extracted
         * b)     and it is not a directory (they are never "extracted")
         * c) or the file name is empty
         */
        if ((!rctx.extract && jcr->impl->last_type != FT_DIREND) ||
            (*jcr->impl->last_fname == 0)) {
          break;
        }
        if (have_acl) {
          /*
           * For anything that is not a directory we delay
           * the restore of acls till a later stage.
           */
          if (jcr->impl->last_type != FT_DIREND) {
            PushDelayedDataStream(rctx, sd);
          } else {
            if (!do_reStoreAcl(jcr, rctx.stream, sd->msg, sd->message_length)) {
              goto bail_out;
            }
          }
        } else {
          non_support_acl++;
        }
        break;

      case STREAM_XATTR_PLUGIN:
      case STREAM_XATTR_HURD:
      case STREAM_XATTR_IRIX:
      case STREAM_XATTR_TRU64:
      case STREAM_XATTR_AIX:
      case STREAM_XATTR_OPENBSD:
      case STREAM_XATTR_SOLARIS_SYS:
      case STREAM_XATTR_DARWIN:
      case STREAM_XATTR_FREEBSD:
      case STREAM_XATTR_LINUX:
      case STREAM_XATTR_NETBSD:
        /*
         * Do not restore Extended Attributes when
         * a) The current file is not extracted
         * b)     and it is not a directory (they are never "extracted")
         * c) or the file name is empty
         */
        if ((!rctx.extract && jcr->impl->last_type != FT_DIREND) ||
            (*jcr->impl->last_fname == 0)) {
          break;
        }
        if (have_xattr) {
          /*
           * For anything that is not a directory we delay
           * the restore of xattr till a later stage.
           */
          if (jcr->impl->last_type != FT_DIREND) {
            PushDelayedDataStream(rctx, sd);
          } else {
            if (!do_restore_xattr(jcr, rctx.stream, sd->msg,
                                  sd->message_length)) {
              goto bail_out;
            }
          }
        } else {
          non_support_xattr++;
        }
        break;

      case STREAM_XATTR_SOLARIS:
        /*
         * Do not restore Extended Attributes when
         * a) The current file is not extracted
         * b)     and it is not a directory (they are never "extracted")
         * c) or the file name is empty
         */
        if ((!rctx.extract && jcr->impl->last_type != FT_DIREND) ||
            (*jcr->impl->last_fname == 0)) {
          break;
        }
        if (have_xattr) {
          if (!do_restore_xattr(jcr, rctx.stream, sd->msg,
                                sd->message_length)) {
            goto bail_out;
          }
        } else {
          non_support_xattr++;
        }
        break;

      case STREAM_SIGNED_DIGEST:
        /*
         * Is this an unexpected signature?
         */
        if (rctx.sig) {
          Jmsg0(jcr, M_ERROR, 0,
                _("Unexpected cryptographic signature data stream.\n"));
          FreeSignature(rctx);
          continue;
        }
        /*
         * Save signature.
         */
        if (rctx.extract && (rctx.sig = crypto_sign_decode(
                                 jcr, (uint8_t*)sd->msg,
                                 (uint32_t)sd->message_length)) == NULL) {
          Jmsg1(jcr, M_ERROR, 0,
                _("Failed to decode message signature for %s\n"),
                jcr->impl->last_fname);
        }
        break;

      case STREAM_MD5_DIGEST:
      case STREAM_SHA1_DIGEST:
      case STREAM_SHA256_DIGEST:
      case STREAM_SHA512_DIGEST:
        break;

      case STREAM_PROGRAM_NAMES:
      case STREAM_PROGRAM_DATA:
        if (!non_support_progname) {
          Pmsg0(000, "Got Program Name or Data Stream. Ignored.\n");
          non_support_progname++;
        }
        break;

      case STREAM_PLUGIN_NAME:
        if (!ClosePreviousStream(jcr, rctx)) { goto bail_out; }
        Dmsg1(50, "restore stream_plugin_name=%s\n", sd->msg);
        if (!PluginNameStream(jcr, sd->msg)) { goto bail_out; }
        break;

      case STREAM_RESTORE_OBJECT:
        break; /* these are sent by Director */

      default:
        if (!ClosePreviousStream(jcr, rctx)) { goto bail_out; }
        Jmsg(jcr, M_WARNING, 0,
             _("Unknown stream=%d ignored. This shouldn't happen!\n"),
             rctx.stream);
        Dmsg2(0, "Unknown stream=%d data=%s\n", rctx.stream, sd->msg);
        break;
    } /* end switch(stream) */
  }   /* end while get_msg() */

  /*
   * If output file is still open, it was the last one in the
   * archive since we just hit an end of file, so close the file.
   */
  if (IsBopen(&rctx.forkbfd)) {
    BcloseChksize(jcr, &rctx.forkbfd, rctx.fork_size);
  }

  if (!ClosePreviousStream(jcr, rctx)) { goto bail_out; }
  jcr->setJobStatus(JS_Terminated);
  goto ok_out;

bail_out:
  jcr->setJobStatus(JS_ErrorTerminated);

ok_out:
#ifdef HAVE_WIN32
  /*
   * Cleanup the copy thread if we restored any EFS data.
   */
  if (jcr->cp_thread) { win32_cleanup_copy_thread(jcr); }
#endif

  /*
   * First output the statistics.
   */
  Dmsg2(10, "End Do Restore. Files=%d Bytes=%s\n", jcr->JobFiles,
        edit_uint64(jcr->JobBytes, ec1));
  if (have_acl && jcr->impl->acl_data->u.parse->nr_errors > 0) {
    Jmsg(jcr, M_WARNING, 0,
         _("Encountered %ld acl errors while doing restore\n"),
         jcr->impl->acl_data->u.parse->nr_errors);
  }
  if (have_xattr && jcr->impl->xattr_data->u.parse->nr_errors > 0) {
    Jmsg(jcr, M_WARNING, 0,
         _("Encountered %ld xattr errors while doing restore\n"),
         jcr->impl->xattr_data->u.parse->nr_errors);
  }
  if (non_support_data > 1 || non_support_attr > 1) {
    Jmsg(jcr, M_WARNING, 0,
         _("%d non-supported data streams and %d non-supported attrib streams "
           "ignored.\n"),
         non_support_data, non_support_attr);
  }
  if (non_support_rsrc) {
    Jmsg(jcr, M_INFO, 0, _("%d non-supported resource fork streams ignored.\n"),
         non_support_rsrc);
  }
  if (non_support_finfo) {
    Jmsg(jcr, M_INFO, 0, _("%d non-supported Finder Info streams ignored.\n"),
         non_support_finfo);
  }
  if (non_support_acl) {
    Jmsg(jcr, M_INFO, 0, _("%d non-supported acl streams ignored.\n"),
         non_support_acl);
  }
  if (non_support_crypto) {
    Jmsg(jcr, M_INFO, 0, _("%d non-supported crypto streams ignored.\n"),
         non_support_crypto);
  }
  if (non_support_xattr) {
    Jmsg(jcr, M_INFO, 0, _("%d non-supported xattr streams ignored.\n"),
         non_support_xattr);
  }

  /*
   * Free Signature & Crypto Data
   */
  FreeSignature(rctx);
  FreeSession(rctx);
  if (jcr->impl->crypto.digest) {
    CryptoDigestFree(jcr->impl->crypto.digest);
    jcr->impl->crypto.digest = NULL;
  }

  /*
   * Free file cipher restore context
   */
  if (rctx.cipher_ctx.cipher) {
    CryptoCipherFree(rctx.cipher_ctx.cipher);
    rctx.cipher_ctx.cipher = NULL;
  }

  if (rctx.cipher_ctx.buf) {
    FreePoolMemory(rctx.cipher_ctx.buf);
    rctx.cipher_ctx.buf = NULL;
  }

  /*
   * Free alternate stream cipher restore context
   */
  if (rctx.fork_cipher_ctx.cipher) {
    CryptoCipherFree(rctx.fork_cipher_ctx.cipher);
    rctx.fork_cipher_ctx.cipher = NULL;
  }
  if (rctx.fork_cipher_ctx.buf) {
    FreePoolMemory(rctx.fork_cipher_ctx.buf);
    rctx.fork_cipher_ctx.buf = NULL;
  }

  if (have_acl && jcr->impl->acl_data) { free(jcr->impl->acl_data->u.parse); }

  if (have_xattr && jcr->impl->xattr_data) {
    free(jcr->impl->xattr_data->u.parse);
  }

  /*
   * Free the delayed stream stack list.
   */
  if (rctx.delayed_streams) {
    DropDelayedDataStreams(rctx, false);
    delete rctx.delayed_streams;
  }

  CleanupCompression(jcr);

  bclose(&rctx.forkbfd);
  bclose(&rctx.bfd);
  FreeAttr(rctx.attr);
}

int DoFileDigest(JobControlRecord* jcr, FindFilesPacket* ff_pkt, bool top_level)
{
  Dmsg1(50, "DoFileDigest jcr=%p\n", jcr);
  return (DigestFile(jcr, ff_pkt, jcr->impl->crypto.digest));
}

bool SparseData(JobControlRecord* jcr,
                SusanWinFilePacket* bfd,
                uint64_t* addr,
                char** data,
                uint32_t* length)
{
  unser_declare;
  uint64_t faddr;
  char ec1[50];

  UnserBegin(*data, OFFSET_FADDR_SIZE);
  unser_uint64(faddr);
  if (*addr != faddr) {
    *addr = faddr;
    if (blseek(bfd, (boffset_t)*addr, SEEK_SET) < 0) {
      BErrNo be;
      Jmsg3(jcr, M_ERROR, 0, _("Seek to %s error on %s: ERR=%s\n"),
            edit_uint64(*addr, ec1), jcr->impl->last_fname,
            be.bstrerror(bfd->BErrNo));
      return false;
    }
  }
  *data += OFFSET_FADDR_SIZE;
  *length -= OFFSET_FADDR_SIZE;
  return true;
}

bool StoreData(JobControlRecord* jcr,
               SusanWinFilePacket* bfd,
               char* data,
               const int32_t length,
               bool win32_decomp)
{
  if (jcr->impl->crypto.digest) {
    CryptoDigestUpdate(jcr->impl->crypto.digest, (uint8_t*)data, length);
  }

  if (win32_decomp) {
    if (!processWin32BackupAPIBlock(bfd, data, length)) {
      BErrNo be;
      Jmsg2(jcr, M_ERROR, 0,
            _("Write error in Win32 Block Decomposition on %s: %s\n"),
            jcr->impl->last_fname, be.bstrerror(bfd->BErrNo));
      return false;
    }
#ifdef HAVE_WIN32
  } else {
    if (bfd->encrypted) {
      if (win32_send_to_copy_thread(jcr, bfd, data, length) !=
          (ssize_t)length) {
        BErrNo be;
        Jmsg2(jcr, M_ERROR, 0, _("Write error on %s: %s\n"),
              jcr->impl->last_fname, be.bstrerror(bfd->BErrNo));
        return false;
      }
    } else {
      if (bwrite(bfd, data, length) != (ssize_t)length) {
        BErrNo be;
        Jmsg2(jcr, M_ERROR, 0, _("Write error on %s: %s\n"),
              jcr->impl->last_fname, be.bstrerror(bfd->BErrNo));
      }
    }
  }
#else
  } else if (bwrite(bfd, data, length) != (ssize_t)length) {
    BErrNo be;
    Jmsg2(jcr, M_ERROR, 0, _("Write error on %s: %s\n"), jcr->impl->last_fname,
          be.bstrerror(bfd->BErrNo));
    return false;
  }
#endif

  return true;
}

/**
 * In the context of jcr, write data to bfd.
 * We write buflen bytes in buf at addr. addr is updated in place.
 * The flags specify whether to use sparse files or compression.
 * Return value is the number of bytes written, or -1 on errors.
 */
int32_t ExtractData(JobControlRecord* jcr,
                    SusanWinFilePacket* bfd,
                    POOLMEM* buf,
                    int32_t buflen,
                    uint64_t* addr,
                    char* flags,
                    int32_t stream,
                    RestoreCipherContext* cipher_ctx)
{
  char* wbuf;     /* write buffer */
  uint32_t wsize; /* write size */
  uint32_t rsize; /* read size */
  char ec1[50];   /* Buffer printing huge values */

  rsize = buflen;
  jcr->ReadBytes += rsize;
  wsize = rsize;
  wbuf = buf;

  if (BitIsSet(FO_ENCRYPT, flags)) {
    if (!DecryptData(jcr, &wbuf, &wsize, cipher_ctx)) { goto bail_out; }
    if (wsize == 0) { return 0; }
  }

  if (BitIsSet(FO_SPARSE, flags) || BitIsSet(FO_OFFSETS, flags)) {
    if (!SparseData(jcr, bfd, addr, &wbuf, &wsize)) { goto bail_out; }
  }

  if (BitIsSet(FO_COMPRESS, flags)) {
    if (!DecompressData(jcr, jcr->impl->last_fname, stream, &wbuf, &wsize,
                        false)) {
      goto bail_out;
    }
  }

  if (!StoreData(jcr, bfd, wbuf, wsize, BitIsSet(FO_WIN32DECOMP, flags))) {
    goto bail_out;
  }
  jcr->JobBytes += wsize;
  *addr += wsize;
  Dmsg2(130, "Write %u bytes, JobBytes=%s\n", wsize,
        edit_uint64(jcr->JobBytes, ec1));

  /*
   * Clean up crypto buffers
   */
  if (BitIsSet(FO_ENCRYPT, flags)) {
    /*
     * Move any remaining data to start of buffer
     */
    if (cipher_ctx->buf_len > 0) {
      Dmsg1(130, "Moving %u buffered bytes to start of buffer\n",
            cipher_ctx->buf_len);
      memmove(cipher_ctx->buf, &cipher_ctx->buf[cipher_ctx->packet_len],
              cipher_ctx->buf_len);
    }
    /*
     * The packet was successfully written, reset the length so that the next
     * packet length may be re-read by UnserCryptoPacketLen()
     */
    cipher_ctx->packet_len = 0;
  }

  return wsize;

bail_out:
  return -1;
}

/**
 * If extracting, close any previous stream
 */
static bool ClosePreviousStream(JobControlRecord* jcr, r_ctx& rctx)
{
  /*
   * If extracting, it was from previous stream, so
   * close the output file and validate the signature.
   */
  if (rctx.extract) {
    if (rctx.size > 0 && !IsBopen(&rctx.bfd)) {
      Jmsg0(rctx.jcr, M_ERROR, 0,
            _("Logic error: output file should be open\n"));
      Dmsg2(000, "=== logic error size=%d bopen=%d\n", rctx.size,
            IsBopen(&rctx.bfd));
    }

    if (rctx.prev_stream != STREAM_ENCRYPTED_SESSION_DATA) {
      DeallocateCipher(rctx);
      DeallocateForkCipher(rctx);
    }

#ifdef HAVE_WIN32
    if (jcr->cp_thread) { win32_flush_copy_thread(jcr); }
#endif

    if (jcr->IsPlugin()) {
      PluginSetAttributes(rctx.jcr, rctx.attr, &rctx.bfd);
    } else {
      SetAttributes(rctx.jcr, rctx.attr, &rctx.bfd);
    }
    rctx.extract = false;

    /*
     * Now perform the delayed restore of some specific data streams.
     */
    if (!PopDelayedDataStreams(jcr, rctx)) { return false; }

    /*
     * Verify the cryptographic signature, if any
     */
    rctx.type = rctx.attr->type;
    VerifySignature(rctx.jcr, rctx);

    /*
     * Free Signature
     */
    FreeSignature(rctx);
    FreeSession(rctx);
    ClearAllBits(FO_MAX, rctx.jcr->impl->ff->flags);
    Dmsg0(130, "Stop extracting.\n");
  } else if (IsBopen(&rctx.bfd)) {
    Jmsg0(rctx.jcr, M_ERROR, 0,
          _("Logic error: output file should not be open\n"));
    Dmsg0(000, "=== logic error !open\n");
    bclose(&rctx.bfd);
  }
  return true;
}

static void FreeSignature(r_ctx& rctx)
{
  if (rctx.sig) {
    CryptoSignFree(rctx.sig);
    rctx.sig = NULL;
  }
}

void FreeSession(r_ctx& rctx)
{
  if (rctx.cs) {
    CryptoSessionFree(rctx.cs);
    rctx.cs = NULL;
  }
}

} /* namespace filedaemon */
